package src;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;



public class RemoveStopWords {

	/**
	 * @param args
	 */
	
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String dir = "C:/Documents and Settings/Administrator/workspace/smsindex/data/dic/";
		String file1 = dir+"originalhindiFAQtfidf";
		String file2 = dir+"hindiStopwords";
		String idffile = dir+"hindiIdf";
		try {
			PrintStream sysout = new PrintStream(System.out, true, "UTF-8");
			BufferedReader b1 = new BufferedReader(new InputStreamReader(
					new FileInputStream(file1), "UTF8"));
			BufferedReader b2 = new BufferedReader(new InputStreamReader(
					new FileInputStream(file2), "UTF8"));
			BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(idffile),"UTF8"));
			
			String read1 = "";
			String read2 = "";
			LinkedList<String> stopWords = new LinkedList();
			while((read2 = b2.readLine())!= null){
				stopWords.add(read2);
			}
			while((read1 = b1.readLine())!= null){
				//sysout.println(read1);
				boolean flag = true;
				String [] token = read1.split(" ");
				//sysout.println("token1 "+token[0]);
				Iterator it = stopWords.iterator();
				while(it.hasNext()){
					//read2.trim();
					read2 = (String)it.next();
					//sysout.println(read2);
					if(token[0].equals(read2)){
						//sysout.println(token[0]+" ::# "+read2);
						flag = false;
					}
				}
				if(flag){
					writer.write(read1);
					writer.newLine();
					//sysout.println(read1);
				}
			}
			b1.close();
			b2.close();
			writer.close();
			
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}catch (IOException e) {
			e.printStackTrace();
		}
		

	}

}
